{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1277938800.0\n",
      "1293753600.0\n",
      "/data/puma1/scratch/malard/export/mtngla_tdx_-1302835696.nc\n"
     ]
    }
   ],
   "source": [
    "import DataSetQuery\n",
    "import datetime\n",
    "\n",
    "\n",
    "environmentName = 'DEV'\n",
    "#Sets the URL of the Malard ServiceGateway.\n",
    "query = DataSetQuery.DataSetQuery('http://localhost:9000',environmentName)\n",
    "\n",
    "fileName = query.executeQuery('mtngla', 'tdx', 400000, 500000, 0, 100000, datetime.datetime(2010, 7, 1, 0, 0), datetime.datetime(2010, 12, 31, 0, 0), [], [])\n",
    "\n",
    "print(fileName)\n",
    "#print(query.getEnvironment(environmentName))\n",
    "\n",
    "#determines the databases that store multiple datasets \n",
    "#print(query.getParentDataSets())\n",
    "#retrieve the data sets for a given parent\n",
    "#print(query.getDataSets('mtngla'))\n",
    "#The overall data cube for the two datasets\n",
    "#print(query.getDataSetBoundingBox( 'mtngla','tandemx' ))\n",
    "#print(query.getDataSetBoundingBox( 'mtngla','srtm' ))\n",
    "\n",
    "#Setup the bounding box\n",
    "minX=-1800000\n",
    "maxX=1300000\n",
    "minY=-1200000\n",
    "maxY=5600000\n",
    "minT=datetime.datetime(2010,7,1,0,0)\n",
    "maxT=datetime.datetime(2010,12,31,23,59)\n",
    "\n",
    "#Queries the catalogue and returns the gridcells grouped by x and y with the min and max times.\n",
    "#gcs = query.getGridCells( 'mtngla','tandemx',minX,maxX,minY,maxY,minT,maxT )\n",
    "#print(gcs)\n",
    "\n",
    "#Queries the catalogue to determine the shards that comprise the bounding box of interest.\n",
    "#shards = query.getShards('mtngla','tandemx',minX,maxX,minY,maxY,minT,maxT)\n",
    "#print(shards)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loads the json output of the getGridCells call and creates a flattened DataFrame.\n",
    "from pandas.io.json import json_normalize\n",
    "import json\n",
    "\n",
    "data = json.loads(gcs)\n",
    "df = json_normalize(data)\n",
    "\n",
    "print(df)\n",
    "\n",
    "totalPoints = df['totalPoints'].sum()\n",
    "maxNumPoints = df['totalPoints'].max()\n",
    "\n",
    "print('TP=[%d] MaxPoints in Cell=[%d]'%(totalPoints,maxNumPoints))   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Converts the Json output of the getShards call into a DataFrame.\n",
    "dataShard = json.loads(shards)\n",
    "dfShards = json_normalize(dataShard)\n",
    "\n",
    "print(dfShards['shardName'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Gets the available column names from the NetCDFs that are in the BoundingBox.\n",
    "print(query.getDataSetColumns('mtngla','tandemx',minX,maxX,minY,maxY,minT,maxT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import MalardHelpers\n",
    "\n",
    "#Returns a file handle to a NetCDF that is a merge and filter of all the shards in a BoundingBox.\n",
    "fileName = query.getNetCdfFile('mtngla','tandemx',minX,maxX,minY,maxY,minT,maxT)\n",
    "\n",
    "print(\"File created: %s\" % (fileName))\n",
    "\n",
    "#Converts the NetCDF into a DataFrame.\n",
    "df = MalardHelpers.getDataFrameFromNetCDF(fileName)\n",
    "\n",
    "print(\"Max Elevation %f\" % (df['elev'].max()))\n",
    "print(\"Min Elevation %f\" % (df['elev'].min()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#a list of columns to see in the output (x,y,time will be added)\n",
    "projections = ['elev','power','coh'] \n",
    "#a list of dictionaries that specify the filters\n",
    "#only numeric fields are supported with the following operations gt(greater than), gte ( greater than equals), lt (less than) and lte (less than equals)\n",
    "#filters are treated as and conditions\n",
    "filters = [{'column':'power','op':'gt','threshold':10000},{'column':'coh','op':'gt','threshold':0.8}]\n",
    "\n",
    "#No filters\n",
    "nofilters = query.executeQuery( 'mtngla','tandemx',minX,maxX,minY,maxY,minT,maxT,projections,[])\n",
    "#Converts the NetCDF into a DataFrame.\n",
    "dfNoFilter = MalardHelpers.getDataFrameFromNetCDF(nofilters)\n",
    "\n",
    "print( \"Max Coh [%f] Min Coh [%f] Max Power [%f] Min Power [%f] Count [%d]\" % ( dfNoFilter['coh'].max(),  dfNoFilter['coh'].min(), dfNoFilter['power'].max(), dfNoFilter['power'].min(), dfNoFilter['power'].count() ) )  \n",
    "\n",
    "#With filters\n",
    "withfilters = query.executeQuery( 'mtngla','tandemx',minX,maxX,minY,maxY,minT,maxT,projections,filters)\n",
    "#Converts the NetCDF into a DataFrame.\n",
    "withFilter = MalardHelpers.getDataFrameFromNetCDF(withfilters)\n",
    "\n",
    "print( \"Max Coh [%f] Min Coh [%f] Max Power [%f] Min Power [%f] Count [%d]\" % ( withFilter['coh'].max(),  withFilter['coh'].min(), withFilter['power'].max(), withFilter['power'].min(), withFilter['power'].count() ) )  \n",
    "\n",
    "print( \"Max Lon [%f] Min Coh [%f] Max Power [%f] Min Power [%f] Count [%d]\" % ( withFilter['x'].max(),  withFilter['coh'].min(), withFilter['power'].max(), withFilter['power'].min(), withFilter['power'].count() ) )  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "## plot with no filters\n",
    "plt.figure(figsize=(12,10))\n",
    "plt.scatter(x=dfNoFilter['x'], y=dfNoFilter['y'], c=dfNoFilter['elev'], marker='.', s=20, vmax=7500)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot with filters\n",
    "plt.figure(figsize=(12,10))\n",
    "#vmin and vmax set min and max of colour bar\n",
    "plt.scatter(x=withFilter['x'], y=withFilter['y'], c=withFilter['elev'], marker='.', s=20, vmax=dfNoFilter['elev'].max(), vmin=dfNoFilter['elev'].min())\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#swathDetails - information about the source swath data and how the DataSetLoader did the gridding.\n",
    "\n",
    "swathById = query.getSwathDetailsFromId('mtngla','tandemx',1)\n",
    "\n",
    "print(swathById)\n",
    " \n",
    "swathByName = query.getSwathDetailsFromName('mtngla','tandemx','CS_LTA__SIR_SIN_2S_20110713T134322_20110713T134453_C001.nc')\n",
    "\n",
    "print(swathByName)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#swathDetails - get all the swaths for a particular data set.\n",
    "import pandas as pd\n",
    "import MalardHelpers as m\n",
    "\n",
    "df = m.getSwathDetailsAsDataFrame('mtngla','tandemx',query)\n",
    "\n",
    "print(df.info())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sourcePath = '/data/snail1/scratch/test/'#put your input path in here\n",
    "fileName = 'testmask.shp'#put your test file in here\n",
    "parentDsName = 'test'\n",
    "dataSet = 'RS6'\n",
    "maskType = 'Glacier' \n",
    "region = 'HMA' \n",
    "minX = 500000 \n",
    "minY = 0 \n",
    "size = 100000 \n",
    "\n",
    "query.publishMask(sourcePath, fileName, parentDsName, dataSet, maskType, region, minX, minY, size )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(query.getMasks('test', dataSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parentdataset = 'test'\n",
    "dataSet = 'RS6'\n",
    "maskType = 'Glacier'\n",
    "region = 'HMA'\n",
    "\n",
    "print(query.getGridCellMasks(parentdataset, dataSet, maskType, region))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gets the mask for a gridcell.\n",
    "\n",
    "print(query.getGridCellMask(parentdataset, dataSet, maskType, region,minX,minY,size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parentDsName = 'test'\n",
    "runName = 'gctest_2'\n",
    "minX = 0.0\n",
    "minY = 200000.0\n",
    "size = 100000.0\n",
    "\n",
    "#expects a dictionary of string to double.\n",
    "statistics = {'NumberOfPoints':1000,'MaskedPoints':500}\n",
    "\n",
    "query.publishGridCellStats( parentDsName, runName, minX, minY, size, statistics )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the available run statistics\n",
    "parentDsName = 'mtngla'\n",
    "query.getAvailableRunStatistics( parentDsName )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the statistics for a gridcell.\n",
    "query.getGridCellStatistics(parentDsName, runName, minX, minY, size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query.getRunStatistics(parentDsName,\"TestRun\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get the proj4 for code for the given shortCode.\n",
    "print( query.getProjection('HMA'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Publish points for a grid cell.\n",
    "parentDataSet = 'mtngla' #intended usage mtngla\n",
    "dataSet = 'publishtest'\n",
    "projection = 'HMA'\n",
    "sourceFileName = '/data/snail1/scratch/tandemx/swath/y2017/m6/cell_HMA_0_600000/GridCell_0.nc'\n",
    "minX = 0\n",
    "minY = 600000\n",
    "size = 100000\n",
    "\n",
    "query.publishGridCellPoints(parentDataSet, dataSet, minX, minY, size, sourceFileName, projection)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(datetime.datetime.fromtimestamp(1337598197))\n",
    "print(datetime.datetime.fromtimestamp(1279356087))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
